package com.shjl.kechuang.utils;

import com.shjl.kechuang.domain.pojo.TechSme;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.jupiter.api.Test;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
 * @title: WebCrawlingUtil
 * @author: wywang
 * @date: 2025/03/31
 * @description: 抓取页面信息
 **/
public class WebCrawlingUtil {

    @Test
    public void test() {
        try {
            List<TechSme> hotSearchList = scrapeTechSme();
            for (TechSme hotSearch : hotSearchList) {

                System.out.println("----------结束了---------------");
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static List<TechSme> scrapeTechSme() throws IOException {
        String url = "https://www.csdn.net/"; // 替换为目标页面URL
        Document doc = Jsoup.connect(url).get();

        List<TechSme> hotSearchList = new ArrayList<>();

        // 根据页面结构选择合适的CSS选择器
        Elements items = doc.getElementsByClass("home-info-banner");// 每条新闻项的根节点
        for (Element item : items) {
            // 提取标题
            Elements elements = item.getElementsByTag("a");

            for (Element element : elements) {
                // 检查是否存在 "banner-img" 类的元素
                Elements bannerImgElements = element.getElementsByClass("banner-img");
                if (bannerImgElements.isEmpty()){
                    continue;
                }
                String style = bannerImgElements.get(0).attr("style");
                String stringUrl = style.substring(style.indexOf("(") + 1, style.lastIndexOf(")"));
                System.out.println(stringUrl);

                String title = element.getElementsByTag("p").text();
                System.out.println(title);
                System.out.println("=====================");
            }

        }

        return hotSearchList;
    }

}
